# This is the place for the python code to do backup
# old and backup code
# maybe useful but I am not sure

'''
# Currently NOT Using, please use step2-method2
# And I have stopped updating this
# OLD but still working method1
# step2-method1: compute the freq of freq for this distribution
# Updated by Wei 2013/04/11
# This method works OK for the fixed ranges and do not want to modify it cause is working OK.
# Instead, I will develop a new alg. which can read the ranges from the plaintext file. :)
print "do step2"
UPPER_BOUND_FOR_RANGE1 = 100
UPPER_BOUND_FOR_RANGE2 = 5000
UPPER_BOUND_FOR_RANGE3 = 80000
UPPER_BOUND_FOR_RANGE4 = 600000
currentTwoDClass = "N/A"

freqOfFreqDict = {}
freqOfFreqDict2DAlsoBasedOnTermDocumentDistribution = {}
freqOfFreqDictWithQueryTermList2DAlsoBasedOnTermDocumentDistribution = {}

classLabelList = ["VR","NF","M","F","VF"]
for i in range(0,20):
    for classLabel in classLabelList:
        key = str(i) + "_" + classLabel
        freqOfFreqDict2DAlsoBasedOnTermDocumentDistribution[key] = 0

queryTermsWithFreqInCollectionDict = {}
inputAuxFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/100KQueryTermsWithTermFreqInCollection.txt"
inputAuxFileHandler = open(inputAuxFileName,"r")
for line in inputAuxFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    freqInCollection = int(lineElements[1])
    if queryTerm not in queryTermsWithFreqInCollectionDict:
        queryTermsWithFreqInCollectionDict[queryTerm] = freqInCollection
    else:
        print "Unexpected Behaviour"

print "len(queryTermsWithFreqInCollectionDict):",len(queryTermsWithFreqInCollectionDict)
inputAuxFileHandler.close()


inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/realFreqOfTermsInQueries_head_85K_0_85%_sortedByQueryTermFreq"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/freqOfFreqInQueries_head_85K_0_85%_sortedByFreqR.txt"
outputFileHandler = open(outputFileName,"w")

outputFileName2 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/freqOfFreqInQueries_head_85K_0_85%_2D_with_query_terms.txt"
outputFileHandler2 = open(outputFileName2,"w")

for i in range(1,11473):
    freqOfFreqDict[i] = 0

print "len(freqOfFreqDict):",len(freqOfFreqDict)

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    freq = int(lineElements[1])
    if freq < 20:
        if queryTerm in queryTermsWithFreqInCollectionDict:
            lengthOfListForLexiconTerm = queryTermsWithFreqInCollectionDict[queryTerm]
            if lengthOfListForLexiconTerm < 1:
                print "The query term:",queryTerm,"does NOT appear in the lexicon."
            else:
                if lengthOfListForLexiconTerm >= 1 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE1:
                    # it is very rare
                    # example:
                    # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                    currentTwoDClass = "VR"
        
                elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE1 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE2:
                    # it is not frequent
                    # example:
                    # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                    currentTwoDClass = "NF"
        
                elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE2 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE3:
                    # it is medium
                    # example:
                    # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                    currentTwoDClass = "M"
        
                elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE3 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE4:
                    # it is frequent
                    # example:
                    # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                    currentTwoDClass = "F"
        
                elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE4:
                    # it is very frequent
                    # example:
                    # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                    currentTwoDClass = "VF"
                
                currentTwoDClass = str(freq) + "_" + currentTwoDClass
                
                print queryTerm,freq,lengthOfListForLexiconTerm,currentTwoDClass
                
                freqOfFreqDict2DAlsoBasedOnTermDocumentDistribution[currentTwoDClass] += 1
                if currentTwoDClass not in freqOfFreqDictWithQueryTermList2DAlsoBasedOnTermDocumentDistribution:
                    freqOfFreqDictWithQueryTermList2DAlsoBasedOnTermDocumentDistribution[currentTwoDClass] = []
                    freqOfFreqDictWithQueryTermList2DAlsoBasedOnTermDocumentDistribution[currentTwoDClass].append(queryTerm)
                else:
                    freqOfFreqDictWithQueryTermList2DAlsoBasedOnTermDocumentDistribution[currentTwoDClass].append(queryTerm)
        else:
            print "The query term:",queryTerm,"does NOT appear in the lexicon."
    freqOfFreqDict[freq] += 1

# output the freq of freq
for i in range(1,11473):
    outputFileHandler.write(str(i) + " " + str( freqOfFreqDict[i] ) + "\n")


print "freqOfFreqDict2DAlsoBasedOnTermDocumentDistribution:",freqOfFreqDict2DAlsoBasedOnTermDocumentDistribution
print "len(freqOfFreqDict2DAlsoBasedOnTermDocumentDistribution):",len(freqOfFreqDict2DAlsoBasedOnTermDocumentDistribution)

for key in freqOfFreqDict2DAlsoBasedOnTermDocumentDistribution:
    outputLine = key + " " + str(freqOfFreqDict2DAlsoBasedOnTermDocumentDistribution[key]) + " "
    
    if key in freqOfFreqDictWithQueryTermList2DAlsoBasedOnTermDocumentDistribution:
        for queryTerm in freqOfFreqDictWithQueryTermList2DAlsoBasedOnTermDocumentDistribution[key]:
            outputLine += queryTerm + " "
    
    outputLine += "\n"
    outputFileHandler2.write(outputLine)


inputFileHandler.close()
outputFileHandler.close()
outputFileHandler2.close()
'''